package org.joy.hpc.analyzer.examples.sogou;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;


/**
 * Sogou 互联网文档输入格式
 * 
 * @author Jeremy Chow(coderplay@gmail.com), Song Liu(lamfeeling@126.com)
 */
//TODO: 应该继承于Writable
public class SogouDocumentInput {

	private static final Charset DEFAULT_CHARSET = Charset.forName("GBK");
	private static final int MAX_DOCUMENT_LENGTH=1024*1024;
	private static final String[] REJECT_SUFFIX = new String[] { "doc", "pdf",
			"mp3", "jpg","avi","bmp","exe","avi","wma","wmv","rar","zip","xls" };

	private Charset charset;

	private String documentId = "";
	private String path = "NO PATH";
	private String title = "NO TITLE";
	private String content = "<HTML><BODY>Initial Content</BODY></HTML>";

	public SogouDocumentInput() {
		this(DEFAULT_CHARSET);
	}

	public SogouDocumentInput(String encoding) {
		this(Charset.forName(encoding));
	}

	private boolean checkSuffix(String path) {
		for (String suffix : REJECT_SUFFIX) {
			if (path.toLowerCase().endsWith("."+suffix)){
				System.err.println("error types"+suffix);
				return false;
			}
		}
		return true;
	}

	/**
	 * use the raw byte[] extracted in sogou text corpus to initialize an
	 * instance
	 * 
	 * @param raw
	 *            the raw bytes
	 * @throws UnsupportedEncodingException
	 */
	public void setRawBytes(byte[] raw) {
		try {
			//test whether the size of doc is too large to parse
			if(raw.length > MAX_DOCUMENT_LENGTH){
				System.err.println("Doc Too Large");
				return;
			}
			// Find the charset first
			final int headerLen = 2048;
			byte[] buf = new byte[headerLen];
			// analyze the first 2048 bytes to find the charset
			System.arraycopy(raw, 0, buf, 0,
					raw.length > headerLen ? buf.length : raw.length);

			String encoding;

			int index = SogouCorpusReader.findBytes(buf, "charset=".getBytes(),
					0,false);
			int end = SogouCorpusReader.findBytes(buf, ">".getBytes(), index,false);
			if (index == -1 || end == -1) {
				// if not found, use the default charset
				encoding = "gb2312";
			} else {
				index += "charset=".length();
				encoding = new String(buf, index, end - index + 1).split("\\s")[0]
						.replaceAll("\'*\"*/*>*", "").trim().replaceAll(
								"[^\\w|-]", "");
			}
			String whole;
			try {
				charset = Charset.forName(encoding);
				// read the whole text using the found charset
				whole = new String(raw, charset);
			} catch (Exception ex) {
				System.err.println("charset error:" + encoding);
				// guess what charset it should be
				if (encoding.toLowerCase().startsWith("u")) {
					encoding = "utf-8";
					charset = Charset.forName(encoding);
					System.out.println("afterFix " + encoding);
				} else if (encoding.toLowerCase().startsWith("g")) {
					encoding = "gbk";
					charset = Charset.forName(encoding);
					System.out.println("afterFix " + encoding);
				} else if (encoding.toLowerCase().startsWith("b")) {
					encoding = "big5";
					charset = Charset.forName(encoding);
					System.out.println("afterFix " + encoding);
				} else {
					// if charset name error, use the default charset.
					charset = Charset.defaultCharset();
				}
				whole = new String(raw, charset);
			}
			BufferedReader r = new BufferedReader(new StringReader(whole));
			// read the fields
			String line = r.readLine();
			while (line != null) {
				boolean contentStarted = false;
				if (line.startsWith("<DOCNO>")) {
					documentId = line.replaceAll("</*DOCNO>", "");
				} else if (line.startsWith("<URL>")) {
					path = line.replaceAll("</*URL>", "");
					// reject tpyes?
					if (!checkSuffix(path)) {
						return;
					}
					line = r.readLine();
					contentStarted = true;
				}
				if (contentStarted) {
					StringBuffer sb = new StringBuffer();
					while (line != null) {
						sb.append(line + "\r\n");
						line = r.readLine();
					}
					content = sb.toString();
				}
				line = r.readLine();
			}
			// FIXME: NOT SUPPORTED BY SOGOU CORPUS
			title = "";
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	public SogouDocumentInput(Charset charset) {
		this.charset = charset;
	}

	/** url */
	public String getURL() {
		return path;
	}

	/** contenttitle */
	public String getTitle() {
		return title;
	}

	/** content */
	public String getHTMLContent() {
		return content;
	}

	/** docno */
	public String getDocumentId() {
		return documentId;
	}
}
